********************************************************************************
*********** This program is written for the present value project **************
********************************************************************************
* 03. VAR Aggregate Analysis Nov2020
********************************************************************************

 

********************************************************************************
* 03.00  Initialize STATA
********************************************************************************
clear all
set more off, perm

********************************************************************************
* 03.01  Settings
********************************************************************************
* Extra state variables in augmented VAR
local basic_vars = "r roe iva bm" 
local extra_vars = "TY DEF VS"

* Sample period
local start = 1965																 
local end = 2022 

* Choose rho
local rho = 0.96 

* Size, MB and Vuolteenaho requirement (1: yes, 0: no)
local size_req = 1
local mb_req = 1
local vuol_req = 0 	

* Number of lags	 
local lags = 1


********************************************************************************
* 03.02  Apply data filters and save temporarily
********************************************************************************
* Load annual dataset
use "Data/created/Annual Data 2021 with Bins", clear 

* Choose sample years
keep if year >= `start'-1 & year <= `end' 

* Default requirements
keep if datareq==1 & ffdata_L1==1

* Optional requirements
if `size_req' == 1 {
	keep if size_L1 == 1
}
if `mb_req' == 1 {
	drop if MB_L1 < 1/100 | MB_L1 > 100 | MB < 1/100 | MB > 100
}	
if `vuol_req' == 1 {
	keep if vuolreq == 1
}   

* Save temporarily
tempfile data
save "`data'"

********************************************************************************
* 03.03  Generate aggregate variables
******************************************************************************** 
* Indicator for missing values
capt drop miss
gen miss = 1 - ~mi(BE_L1) & ~mi(MB_L1) & ~mi(MB) & ~mi(BE) & ~mi(ROE) ///
			& ~mi(R) & ~mi(D) & ~mi(N_dilute) & ~mi(Y) & ~mi(ME)

* MB: Use book equity as the weight
g MB_L1_Den = BE_L1 if ~mi(MB_L1)
g MB_L1_Num = MB_L1 * BE_L1
g MB_Den = BE if ~mi(MB)
g MB_Num = MB * BE
	
* ROE: Use one lag of book equity as a weight
g ROE_Den = BE_L1 if ~mi(ROE)
g ROE_Num = ROE * BE_L1 
	
* Return: Use one lag of market equity as a weight
g R_Den = ME_L1 if ~mi(R)
g R_Num = R * ME_L1
	
* Scale
g SCALE_Num1 = D
g SCALE_Num2 = BE
g SCALE_Num3 = N_dilute * ME
g SCALE_Den1 = ME if ~mi(N_dilute)
g SCALE_Den2 = BE_L1 + Y if ~mi(BE) & ~mi(N_dilute) & ~mi(D)

* Plowback
gen LAMBDA_Num = N_dilute * BE 
gen LAMBDA_Den = D + N_dilute * BE

* Market portfolio 
collapse (sum) MB_L1_Den MB_L1_Num MB_Den MB_Num ROE_Den ROE_Num R_Den R_Num ///
	SCALE_Num* SCALE_Den*, by(year)
	
tsset year	
	
g MB_L1 = MB_L1_Num/MB_L1_Den
g MB = MB_Num/MB_Den
g ROE = ROE_Num/ROE_Den
g SCALE = (SCALE_Num1 + SCALE_Num2 * (SCALE_Num3 / SCALE_Den1)) / SCALE_Den2
g R = R_Num/R_Den
g LAMBDA = ((SCALE_Num3 / SCALE_Den1) * SCALE_Num2) / (SCALE_Num1 + (SCALE_Num3 / SCALE_Den1) * SCALE_Num2)

g mb_L1 = log(MB_L1)
g mb = log(MB)
g bm = -mb
g bm_L1 = -mb_L1
g r = log(1+R)
g roe = log(1+ROE)
g iva = log(SCALE)

* Check the exact nonlinear identity 
gen R_err = (1+ROE) * SCALE * (1 + (MB - 1) * LAMBDA) / MB_L1 - (1+R)
summarize R_err, detail

* Lag variables
tsset year
foreach var of varlist r roe iva {
	g `var'_L1 = l1.`var'
}

tempfile mkt
save "`mkt'"
 
********************************************************************************
* 03.04  State variables 
******************************************************************************** 
* 1. Credit spread

	* Moody's AAA Bond (St. Louis Fed)
	import excel "Data/other state variables/AAA.xls", clear ///
		cellrange(A11:B1254) firstrow
		tempfile AAA
		save "`AAA'"

	* Moody's BAA Bond (St. Louis Fed)
	import excel "Data/other state variables/BAA.xls", clear ///
		cellrange(A11:B1254) firstrow

	* Merge and Construct Default Spread (DEF) 
	merge 1:1 observation_date using "`AAA'", nogen
	qui g DEF = BAA - AAA

	* Reformat Date 
	gen ym = ym(year(observation_date), month(observation_date))
		format ym %tm 
	format ym %tm

	* Convert to annual variables 
	keep if month(dofm(ym)) == 6

	* Save Data 
	gen year = year(dofm(ym))
	keep year DEF 
	tempfile DEF
	save "`DEF'"

* 2. Term spread

	* 10-year Constant Maturity Taxable Bond 
	import excel "Data/other state variables/USA 10 year yield - IGUSA10D.xlsx", sheet("Price Data") firstrow clear

	* Initialize Variables 
	keep Date Close
	ren Close ten_year

	* Reformat Date 
	qui g month = substr(Date,1,2)
	qui g day = substr(Date,4,2)
	qui g year = substr(Date,-4,4)
	destring year, replace
	destring month, replace
	destring day, replace

	qui g date = mdy(month,day,year)
	qui g ym = ym(year,month)
	format ym %tm
	order ym

	drop month year day Date

	* Keeping Month-end Obs 
	sort date
	collapse (last) ten_year, by(ym)

	* Save Data 
	tempfile ten_year
	save "`ten_year'"

	* 3-month Constant 
	import excel "Data/other state variables/USA 3 month yield - ITUSA3CMD.xlsx", sheet("Price Data") firstrow clear

	* Initialize Variables
	keep Date Close
	ren Close three_month

	* Reformat Date 
	qui g month = substr(Date,1,2)
	qui g day = substr(Date,4,2)
	qui g year = substr(Date,-4,4)
	destring year, replace
	destring month, replace
	destring day, replace

	qui g date = mdy(month,day,year)
	qui g ym = ym(year,month)
	format ym %tm
	order ym

	drop month year day Date

	* Keeping Month-end Obs 
	sort date
	collapse (last) three_month, by(ym)

	* Save Data 
	tempfile three_month
	save "`three_month'"

	* 1-year Note 
	import excel "Data/other state variables/USA 1 year yield - IGUSA1D.xlsx", sheet("Price Data") firstrow clear

	* Initialize Variables 
	keep Date Close
	ren Close one_year

	* Reformat Date 
	qui g month = substr(Date,1,2)
	qui g day = substr(Date,4,2)
	qui g year = substr(Date,-4,4)
	destring year, replace
	destring month, replace
	destring day, replace

	qui g date = mdy(month,day,year)
	qui g ym = ym(year,month)
	format ym %tm
	order ym

	drop month year day Date

	* Keeping Month-end Obs 
	sort date
	collapse (last) one_year, by(ym)

	* Merge Ten-year Data and Construct TY 
	merge 1:1 ym using "`ten_year'", keep(2 3) nogen
	merge 1:1 ym using "`three_month'", keep(2 3) nogen

	qui g TY = ten_year - one_year
	qui g TY2 = ten_year - three_month

	label var TY "Ten-year minus one-year"
	label var TY2 "Ten-year minus three-month"

	* Restrict Sample and Save Data 
	keep if month(dofm(ym))==6
	g year = year(dofm(ym))
	keep year TY TY2

	tempfile TY
	save "`TY'"

* 3. Price-to-earnings ratio (Shiller(2000))
import excel "Data/other state variables/Shiller_CAPE.xls", ///
	sheet("Data") cellrange(A8:M2423) firstrow clear

	* Keeping Valid Observations 
	keep if Date != .
	keep Date CAPE
	ren CAPE PE

	* Reformat Date 
	qui g year = int(Date)
	qui g month = mod(Date*100, 100) // October issue
	qui g ym = ym(year,month)
	order ym 
	format ym %tm

	* Restrict Sample (1928:12 ~) 
	keep if month==6

	* Destring PE and log transformation 
	destring PE, replace force
	replace PE = ln(PE)

	* Save Data 
	keep year PE
	tempfile PE
	save "`PE'"

* 4. Value spread
import delimited "Data/ff/6_Portfolios_2x3.csv", ///
	varnames(1) rowrange(4843:5994) clear  

	*  Initialize Variables 
	ren thisfilewascreatedbycmpt_me_beme Date
	ren v2 small_low_bm
	ren v4 small_hi_bm

	local var Date small_low_bm small_hi_bm
	keep `var' 
	foreach v in `var'{	
		destring `v', replace
	} 
	qui g month = mod(Date, 100)
	qui g year = int(Date/100)

	* Construct VS at the end of June of year t (BE/ME measured at the end of Dec t-1) 
	qui g VS = ln(small_hi_bm) - ln(small_low_bm)
	keep if month==6	
	keep year VS 

	* Save Data for the Spread at June 
	tempfile VS
	save "`VS'"

	* Merge all data
	u "`mkt'", clear
		merge 1:1 year using "`TY'", keep(1 3) nogen
		merge 1:1 year using "`DEF'", keep(1 3) nogen
		merge 1:1 year using "`PE'", keep(1 3) nogen
		merge 1:1 year using "`VS'", keep(1 3) nogen 
	
	* Lag state variables 
	tsset year
		g TY_L1 = l1.TY
		g DEF_L1 = l1.DEF
		g PE_L1 = l1.PE
		g VS_L1 = l1.VS	
		
	* Sample period
	keep if year >= `start'
	
	* Save data
	tempfile data
	save "`data'"
 
	* Save the data permanently
	save "Data/Created/Annual Data Aggregate Market 2021", replace
 


********************************************************************************
* 03.05  Summary stats   (Table 03_05)
******************************************************************************** 
local count = 1
foreach var of varlist r roe iva mb TY DEF VS {
	preserve
	
	* Equal-weight all years
	collapse (count) N = `var' (mean) mean = `var' (sd) sd = `var' (min) min = `var' ///
			 (p1) p1 = `var' (p25) p25 = `var' (p50) p50 = `var' (p75) p75 = `var' (p99) p99 = `var' (max) max = `var'
	
	* Save stats
	qui tostring N, force replace format("%15.0fc")
	qui tostring mean-max, force replace format("%5.3f")
	g var = "`var'"
	order var
	tempfile tmp`count'
	save "`tmp`count''"
	
	restore
	local count = `count' + 1
	}

* Append all stats 
u "`tmp1'", clear
forval i = 2/7{
	append using "`tmp`i''"
	}
	
* Label variabels 
replace var = "$ r_{t}$" 		if var == "r"
replace var = "$ mb_{t}$" 		if var == "mb"
replace var = "$ roe_{t}$" 		if var == "roe"
replace var = "$ iva_{t}$" 		if var == "iva"  
replace var = "$ TY_{t}$" 		if var == "TY"
replace var = "$ DEF_{t}$" 		if var == "DEF"
replace var = "$ VS_{t}$" 		if var == "VS"  
replace var = "\addlinespace " + var if _n > 1

* Export to Latex 
listtab * using "Results/Tables/03_05_`end'.tex", ///								
		rstyle(tabular) replace ///															
		head("\begin{tabular}{l c c c c c c c c c c}" ///									
		"\midrule" ///
		"Variable & N & Mean & St. Dev. & Min & 1\% & 25\% & Median & 75\% & 99\% & Max \\" ///
		"\midrule") ///
		foot("\midrule" "\end{tabular}")	



********************************************************************************
* 03.06  Begin the VAR analysis 
******************************************************************************** 
use "`data'", clear 

	* Choose state variables	
	local state_vars = "`basic_vars' `extra_vars'"
			 
	* Choose all variables including lags
	local all_vars = "`state_vars'" 
	foreach var in `state_vars' {
		forval i = 1/`lags' {
			local all_vars = "`all_vars' `var'_L`i'" 
		}
	}	 
			
	* Count the total number of variables   
	local num_vars = 0 
	foreach var in `state_vars' {
		local num_vars = `num_vars' + `lags'
	} 
		
	* Define matrix to store the results
	matrix B = J(1,`num_vars',.)													 
	matrix SE = J(1,`num_vars',.)													 
	matrix Rsq = J(1,1,.)												 
	matrix F = J(1,1,.)
	 
	* Define regressors and set of variables
	local regressors = "" 														
	foreach var in `state_vars' {
		forval i = 1/`lags' {
			local regressors = "`regressors' `var'_L`i'" 
		}
	} 
 
****************************************************************************
* 03.07  Demean the variables in the time series
**************************************************************************** 
foreach var in `state_vars' `regressors' {
	egen mean = mean(`var')
	replace `var' = `var' - mean
	drop mean
} 

****************************************************************************
* 03.08  Regress state variables (Table 03_08) 
**************************************************************************** 
local i = 1
foreach var of varlist `state_vars' {
			
	* Regression
	reg `var' `regressors', nocon 
			 
	* Compute error terms				
	predict u_`var', residual
					
	* Store coefficients and standard errors						
	matrix B = (B \ (e(b)))		
	matrix V = vecdiag(e(V))
	matmap V V, map(sqrt(@))
	matrix SE = (SE \ V)
	matrix Rsq = (Rsq \ (e(r2)))
	matrix F = (F \ Ftail(1,45,e(F)))
	
	local i = `i' + 1	
} 
	
* Remove first row
mat B = B[2...,1...]
mat Rsq = Rsq[2...,1...]
mat F = F[2...,1...]
mat SE = SE[2...,1...]
	
* Save data on residuals
tempfil resid
save "`resid'"

* Create a table of coefficients
* Import stored coefficients and standard errors 
clear 
svmat B
svmat Rsq
svmat F
svmat SE
 
* Standard erros 
forval i = 1/`num_vars'{ 
	qui tostring SE`i', force replace format("%5.3f")
	qui replace SE`i' = "(" + SE`i' + ")"
} 

* Make a column for variable names	
g col = ""
local j = 1
foreach var in `state_vars' {
	if "`var'"=="iva" {
		replace col = "$ 0`j'iva_{t} $" if [_n]==`j' 
	}
	else {
		replace col = "$ 0`j'`var'_{t} $" if [_n]==`j'
	}
	local j = `j' + 1
}  
replace col = "\addlinespace " + col if [_n]>=2
order col
foreach var of varlist B* Rsq1 F1 {
	qui tostring `var', force replace format("%5.3f")
}	
tempfile tmp
save "`tmp'"

* Put t-stats below coefficients		
keep col SE*
forval i = 1/`num_vars' {
	ren SE`i' B`i'
	}
append using "`tmp'"
gsort col -SE1
replace col = "" if mi(SE1)

* Rename columns
local j = 1
foreach var in `state_vars' {
	if "`var'"=="iva" {
		replace col = "\addlinespace $  iva_{t} $" if [_n]==`j'									
	}
	else { 
		replace col = "\addlinespace $  `var'_{t} $" if [_n]==`j'	
	}
	local j = `j' + 2
} 
drop SE*

* Export to Latex 
local columns = ""
local c = "c c"
local total_vars = "`state_vars'" 
foreach var in `total_vars' {
	forval i = 1/`lags' {
		if "`var'"=="iva" {
			local columns = "`columns' & $ iva_{t-`i'} $"
		}
		else {
			local columns = "`columns' & $ `var'_{t-`i'} $"
		}
		local c = "`c' c"
		}
	}
local columns = "`columns' & $ R^{2} $ & $ p(F) $" 

* Transition matrix 
listtab * using "Results/Tables/03_08_`end'.tex", ///
	rstyle(tabular) replace ///
	head("\begin{tabular}{l`c'}" ///
	"\midrule" ///
	"`columns' \\" ///
	"\midrule") ///
	foot("\midrule" "\end{tabular}") 
		
****************************************************************************
* 03.09  Compute variance-covariance matrix of residuals (Table 03_09)
**************************************************************************** 		 
local j = 0
local varlist `state_vars'  
foreach var1 in `varlist' {
	local j = `j' + 1
	local k = 0
	foreach var2 in `varlist' {
		local k = `k' + 1
			
		* Average of u_`var1' and u_`var2'   
		u "`resid'", clear  
		egen mean1 = mean(u_`var1') 
		egen mean2 = mean(u_`var2') 
			 
		* Compute variance (covariance) and save locally 
		qui g cov_`j'_`k' = (u_`var1'-mean1) * (u_`var2'-mean2)  
		collapse (mean) cov_`j'_`k'  
		local cov_`j'_`k' = cov_`j'_`k'[1]  
	}
} 
* Export to latex
clear
svmat B
forval i = 1/`num_vars' {
	forval j = 1/`num_vars' {
		replace B`i' = `cov_`i'_`j'' if [_n]==`j'
	}
}
mkmat B*, mat(Sigma) 
g col = ""
local i = 1
foreach var in `state_vars' {
	if "`var'"=="iva" {
		replace col = " $ iva_{t} $" if [_n]==`i'
	}
	else {
		replace col = " $ `var'_{t} $" if [_n]==`i'
	}
	local i = `i' + 1
}

* Keep only state variables
replace col = "\addlinespace " + col if [_n]>=2
order col
foreach var of varlist B* {
	qui tostring `var', force replace format("%5.3f")
} 
local columns = ""
local c = "c" 
foreach var in `state_vars' {
	if "`var'"=="iva" {
		local columns = "`columns' & $ iva_{t} $"
	}
	else {
		local columns = "`columns' & $ `var'_{t} $"
	}
	local c = "`c' c"
}	
			
listtab * using "Results/Tables/03_09_`end'.tex", ///
	rstyle(tabular) replace ///
	head("\begin{tabular}{l`c'}" ///
	"\midrule" ///
	"`columns' \\" ///
	"\midrule") ///
	foot("\midrule" "\end{tabular}")		

****************************************************************************
* 03.10  Decomposition of return news (Table 03_10)
**************************************************************************** 	 

* Define elementary vectors  
forvalues i = 1/3 {
	matrix e`i' = J(`num_vars',1,0) 
	matrix e`i'[`i',1] = 1 
} 

* Generate lambdas 
matrix lambda =	(e1'*`rho'*B*inv(I(`num_vars')-`rho'*B))'
matrix lambda2 = (e2'*inv(I(`num_vars')-`rho'*B))'						// For computing roe news explicitly
matrix lambda3 = (e3'*inv(I(`num_vars')-`rho'*B))'						// For computing iva news explicitly

* Generate news variance and covariance
* Back out everything explicitly
* Variances 
matrix var_N_DR = lambda'*Sigma*lambda
matrix var_N_roe = lambda2'*Sigma*lambda2
matrix var_N_iva = lambda3'*Sigma*lambda3

* Covariances	
matrix cov_N_DR_roe = lambda'*Sigma*lambda2
matrix cov_N_DR_iva = lambda'*Sigma*lambda3
matrix cov_N_roe_iva = lambda2'*Sigma*lambda3	

* Variance-covariance matrix	
local var var_N_DR var_N_roe var_N_iva cov_N_DR_roe cov_N_DR_iva cov_N_roe_iva
foreach mat in `var' {
	local `mat' = `mat'[1,1]
}
matrix totalvar = var_N_DR + var_N_roe + var_N_iva - 2*(cov_N_DR_roe + cov_N_DR_iva - cov_N_roe_iva)
local totalvar = totalvar[1,1]
local svarDR = `var_N_DR' / `totalvar'
local svarroe = `var_N_roe' / `totalvar'
local svariva = `var_N_iva' / `totalvar'
local svarDRroe = -2*`cov_N_DR_roe' / `totalvar'
local svarDRiva = -2*`cov_N_DR_iva' / `totalvar'
local svarroeiva = 2*`cov_N_roe_iva' / `totalvar'			
local var var_N_DR var_N_roe var_N_iva
foreach mat in `var' {
	local `mat' = sqrt(`mat'[1,1])
}
local cov_N_DR_roe = `cov_N_DR_roe'/(`var_N_DR'*`var_N_roe')
local cov_N_DR_iva = `cov_N_DR_iva'/(`var_N_DR'*`var_N_iva')
local cov_N_roe_iva = `cov_N_roe_iva'/(`var_N_roe'*`var_N_iva')	

mat VCM = (`var_N_DR', 0, 0, 0, `svarDR', 0, 0 \ ///
		   `cov_N_DR_roe', `var_N_roe', 0,0, `svarDRroe', `svarroe', 0 \ ///
		   `cov_N_DR_iva', `cov_N_roe_iva', `var_N_iva', 0, `svarDRiva', `svarroeiva', `svariva')

* Export to Latex 
clear
svmat VCM
g col = ""
g row = 0
local i = 1
foreach var in "N_{DR}" "N_{roe}" "N_{iva}" {
		replace col = "$ `var' $" if [_n]==`i'
		replace row = `i'-0.5 if [_n]==`i'
		local i = `i' + 1
		}
replace col = "\addlinespace " + col if [_n]>=2
order col 

*			append using "Output/news_decomp_stderr_C.dta"	

forval i= 1/7 {
	qui tostring VCM`i', force replace format("%5.3f")
	qui replace VCM`i' = "" if VCM`i' == "0.000" 
}

sort row
drop row

* Output to latex
listtab * using "Results/Tables/03_10_`end'.tex", ///
	rstyle(tabular) replace ///
	head("\begin{tabular}{lccccccc}" ///
	"\midrule") ///
	foot("\midrule" "\end{tabular}")
	
